package crawler;


import helper.IOFile;

import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLConnection;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.GregorianCalendar;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.htmlcleaner.CleanerProperties;
import org.htmlcleaner.HtmlCleaner;
import org.htmlcleaner.SimpleHtmlSerializer;
import org.htmlcleaner.TagNode;

import databaseQueries.DatabaseQueries;

import utilities.UtilityKindKeyword;

import entity.Channel;
import entity.ScheduleChannel;
import entity.ScheduleDetail;

/*
 * 1. Crawler data on http://vtv.vn/TVSchedule/136/1/6/1/2014.vtv
 * 2. Extract TV schedule 
 * */
public class CrawlerVTVCab {
	
	private final String CONFIG_FILE = "data/cabVN/Config.txt";
	
	private String DES_FOLDER;
	private String CHANEL_FILE_NAME;
	private String INFOR_FILE_NAME;
	
	// input schedule infor
	
	private  String PUBLISHER;
	private  String URL_SCHEDULE_PAGE;
	private  String TEMP_URL_CHANEL;
	private  String SUFIX ;
	private  String TAG_HTML;
	private  String TOKEN_HTML;
	
	private  ArrayList<Channel> arrayChanel ;

	private IOFile ioFile;
	
	private UtilityKindKeyword kindKeyword;
	
	public CrawlerVTVCab() {
		/* these comments use when run on FSoft desktop to pass proxy */
		/*
		System.getProperties().put("http.proxyHost", "HL-proxyA");
		System.getProperties().put("http.proxyPort", "8080");
		System.getProperties().put("http.proxyUser", "antq");
		System.getProperties().put("http.proxyPassword", "0123999123");
		*/
		ioFile = new IOFile();
		getConfig();
		arrayChanel  = new ArrayList<Channel>();
		kindKeyword = new UtilityKindKeyword();
		
	}
	/*
	 * Step -1: read config data
	 * */
   public void getConfig(){
	   ArrayList<String> arrayCofig = ioFile.readFileToArray(CONFIG_FILE, -1);
	   DES_FOLDER = arrayCofig.get(0).split(": ")[1].trim();
	   CHANEL_FILE_NAME = arrayCofig.get(1).split(": ")[1].trim();
	   INFOR_FILE_NAME = arrayCofig.get(2).split(": ")[1].trim();
   }
	/*
	 * Step 0: read  schedule page information 
	 * */
	public boolean getSchedulePageInfor(){
		ArrayList<String> arrayContent = ioFile.readFileToArray(INFOR_FILE_NAME, -1);
		 int countField = 0;
		try{
		    countField = Integer.parseInt(arrayContent.get(0));
		    PUBLISHER = arrayContent.get(1).split(": ")[1].trim();
			URL_SCHEDULE_PAGE = arrayContent.get(2).split(": ")[1].trim();
			TEMP_URL_CHANEL = arrayContent.get(3).split(": ")[1].trim();
			SUFIX = arrayContent.get(4).split(": ")[1].trim();
			TAG_HTML = arrayContent.get(5).split(": ")[1].trim();
			TOKEN_HTML = arrayContent.get(6).split(": ")[1].trim();
			//System.out.println(PUBLISHER+"\n"+URL_SCHEDULE_PAGE);
		}catch(Exception ex){
			return false;
		}
		if(arrayContent.size() < countField+1 ){
			return false;
		}
		
		return true;
	}
	/*end Step 0: read  schedule page information */
	/*
	 *  Step1: read channel URL to crawl from txt file
	 *  
	 *  */
	 public void getChannels(){
 	    //1. Get time now
 	   GregorianCalendar gCalendar = new GregorianCalendar();
 	   int nowYear = gCalendar.get(Calendar.YEAR);
 	   int nowMonth = gCalendar.get(Calendar.MONTH) +1;
 	   int nowDate = gCalendar.get(Calendar.DATE);
 	   String nowDayString = nowDate +"/"+ nowMonth+"/" + nowYear; 
	    	//2. lay danh sach cac kenh
	    	String contentChannel = ioFile.readFile(CHANEL_FILE_NAME);
	    	String[] arrayContentChannel = contentChannel.split("\n");
	    	for(int i= 1;i< arrayContentChannel.length;i+= 3){
	          	String nameChannel = arrayContentChannel[i];
	          	String kindChannel = arrayContentChannel[i+1];
	          	// template of URL Channel: Regax URL + Day + Sufix, ex: http://vtv.vn/TVSchedule/136/1/24/2/2014.vtv
	          	String URLChannel = arrayContentChannel[i+2]+ nowDayString+ SUFIX;
	          	String publisher = 	PUBLISHER;
	          	Channel ch = new Channel(nameChannel,kindChannel,URLChannel,publisher);
	          	arrayChanel.add(ch);
	    	}
	    }
	 public void storeListChannelDatabase(){
		 for(int i=0;i< arrayChanel.size();i++){
			 Channel ch = arrayChanel.get(i);
			 DatabaseQueries dbq = new DatabaseQueries();
			 if(dbq.checkChannelNotExist(ch.getName(), ch.getPublisher())){
				 dbq.insertTableChannel(ch.getName(), ch.getURL_Schedule(), ch.getKind(), ch.getPublisher(), ch.getDescription());
				 
			 }
		 }
	 }
	/* end Step 1: get channel URL to crawl */
	 
	/* 
	 * Step 2: crawl data from Internet 
	 * 
	 * */
	public void crawl(Channel channel) {
		// crawl lich chieu cho 1 kenh xac dinh(VTV1,HBO,..)
		String result = "";
		try {
			CleanerProperties props = new CleanerProperties();
			// set some properties to non-default values
			props.setTranslateSpecialEntities(true);
			props.setTransResCharsToNCR(true);
			props.setOmitComments(true);
			// clean html tag : automatic repair error tags
			TagNode rootNode = new HtmlCleaner(props).clean(new URL(channel.getURL_Schedule()));
			/* get all tag in html file *** */
			TagNode[] items = rootNode.getAllElements(true);
			//System.out.println("sssssd " + items.length);
			for (TagNode item : items) {
				if (item.getName().equals(TAG_HTML) 
						&& item.getAttributes().toString().equals(TOKEN_HTML)) {
					result += nomalizeText(item.getText().toString());
				}
			}
			
		//	System.out.println(result);
			//Save File
			if(result!= ""){
				ioFile.Write2File(DES_FOLDER+ "/schedule/Schedule_"+channel.getName()+".txt", result, false);
				//System.out.println("Save File "+channel.getName()+".txt"+" Sucess!");
			}
		} catch (Exception ex) {
			ex.printStackTrace();
		}
	}
	
	public void processCrawl(){
		if(checkURLScheduleConnection(getURL_SCHEDULE_PAGE())){
			System.out.println("Check connection OK");
			// thu thap lich chieu cua tung kenh
			System.out.println("Crawling....");
			for(int i=0;i < arrayChanel.size();i++){
				Channel ch = arrayChanel.get(i);
				crawl(ch);
			}	
			System.out.println("Crawl Success!");
		}else{
			System.out.println("URL Die");
		}
	}
	/*end Step 2: crawl data from internet*/
	
	/* 
	 * Step 3:  get TV Schedule from txt data file
	 * 
	 * */
	    /*** set kind functions***/
	public String setProgramKind(Channel channel,String programName){
		String kind = "";
		//neu la kenh chuyen biet thi the loai cua chuong trinh laf the loai cua kenh
		if(channel.getKind().equals("UNION") ){ //UNION : kenh tong hop
			kind = checkProgramKindByKey(programName);		
		}else{
			kind = channel.getKind();
		}
		return kind;
	}
	
	public String checkProgramKindByKey(String programName){
		String kind = "";
		String name = programName.toLowerCase().trim();
		boolean conti = true;  // continue compare ??
		ArrayList<String> arrayMovieKey =  kindKeyword.getArrayKeyMovie();
		for(int i = 0; i<arrayMovieKey.size();i++){
			String key = arrayMovieKey.get(i).toLowerCase();
			if(name.indexOf(key )>=0){
				kind = "MOVIE";
				conti = false;
				break;
			}
		}
		if(conti == true){
			ArrayList<String> arrayMusicKey = kindKeyword.getArrayKeyMusic();
			for(int i = 0; i< arrayMusicKey.size();i++){
				String key = arrayMusicKey.get(i).toLowerCase();
				if(name.indexOf(key)>=0){
					kind = "MUSIC";
					conti = false;
					break;
				}
			}
		}
		if(conti == true){
			ArrayList<String> arraySportKey = kindKeyword.getArrayKeySport();
			for(int i = 0; i< arraySportKey.size();i++){
				String key = arraySportKey.get(i).toLowerCase();
				if(name.indexOf(key)>=0){
					kind = "SPORT";
					conti = false;
					break;
				}
			}
		}
		if(conti == true){
			ArrayList<String> arrayNewsKey = kindKeyword.getArrayKeyNews();
			for(int i = 0; i< arrayNewsKey.size();i++){
				String key = arrayNewsKey.get(i).toLowerCase().trim();
				if(name.indexOf(key)>=0){
					kind = "NEWS";
					conti = false;
					break;
				}
			}
		}
		if(conti == true){
			ArrayList<String> arraySciencesKey = kindKeyword.getArrayKeySciences();
			for(int i = 0; i< arraySciencesKey.size();i++){
				String key = arraySciencesKey.get(i).toLowerCase();
				if(name.indexOf(key)>=0){
					kind = "SCIENCES";
					conti = false;
					break;
				}
			}
		}
		if(conti == true){
			ArrayList<String> arrayGameShowKey = kindKeyword.getArrayKeyGameShow();
			for(int i = 0; i< arrayGameShowKey.size();i++){
				String key = arrayGameShowKey.get(i).toLowerCase();
				if(name.indexOf(key)>=0){
					kind = "GAME SHOW";
					conti = false;
					break;
				}
			}
		}
		if(conti == true){
			kind = "OTHER";
		}
		return kind;
	}
	   /*** end set kind functions ***/
	
	public ScheduleChannel getScheduleForChannel(Channel channel){
		System.out.println(channel.getName());
		/*lay lich chieu cho 1 kenh nhaat dinh tu file text */
		ScheduleChannel scheduleChanel = null;
		//1. read all line in file channel schedule
		ArrayList<String> scheduleData = ioFile.readFileToArray("data/cabVN/schedule/Schedule_"+channel.getName()+".txt",-1);
		String regexDate = "(([012][0-9])||([3][01]))[/](([0][0-9])||[1][0-2])[/](19||20)[0-9][0-9]";
		String regexTime = "([0-9])[h:]([0-9][0-9])";
		Pattern patternDate = Pattern.compile(regexDate );
		Pattern patternTime = Pattern.compile(regexTime );
		String dateShow = "";
		String dateShowNext = "";
		
		ArrayList<ScheduleDetail> arrScheduleDetail = new ArrayList<ScheduleDetail>();
		//check số dòng hợp lệ trong file scheduleData text
		int breakErrosLine =0;
		for(int k = scheduleData.size()-1; k >=0; k--){
			Matcher matcher = patternDate.matcher(scheduleData.get(k));
			Matcher matcherNext = patternDate.matcher(scheduleData.get(k-1));
			if(matcher.find() && !matcherNext.find()){
				System.out.println(matcher.group());
				breakErrosLine =k;
				break;
			}
		}
		int sizeValidSchedule = breakErrosLine ;
		System.out.println(sizeValidSchedule);
		int i=0;
		while(i< sizeValidSchedule){
			  Matcher matcher = patternDate.matcher(scheduleData.get(i));
              if(i== 0 && matcher.find() ){ //dong dau tien la date Time
            	  //System.out.println(scheduleData.get(i));
            	  dateShow = scheduleData.get(i);
            	  dateShowNext = dateShow;
            	  i = i+1;
              }
              if(dateShowNext != dateShow){
            	 dateShow = dateShowNext;
              }
              String startTime = nomalizeText(scheduleData.get(i));
              i = i+1; // doc dong tiep theo
              if(i > sizeValidSchedule){
            	  break;
              }
              String program = nomalizeText(scheduleData.get(i));
              i= i+1; // doc dong tiep theo
              if(i > sizeValidSchedule){
            	  break;
              }
              //  read detail Program 
              String detail = "";
              Matcher matcherTime = patternTime.matcher(scheduleData.get(i));
              // kiem tra truowng hop truong detail trong file txt bi ro~ng.
              if(matcherTime.find()){
            	  detail = "null";
              }else{
                 detail = scheduleData.get(i);
                 i= i+1; // doc dong tiep theo
                 if(i > sizeValidSchedule){
               	  break;
                 }
              }
              //1. doc finish time
              String finishTime = "";
              if(i == sizeValidSchedule){ //da doc het cac dong co nghia trong file
            	  finishTime = "24h00";
              }
              else{
            	//  System.out.println(" sizeValidSchedule "+ sizeValidSchedule+" bug "+ i);
            	  Matcher matcherDate = patternDate.matcher(scheduleData.get(i));
            	  if(matcherDate.find()){
            		  dateShowNext = scheduleData.get(i);
                	  i = i+1;
                	  if(i > sizeValidSchedule){
                    	  break;
                      }
                	  finishTime = "24h00";
            	  }
                  else{
            		  finishTime = nomalizeText(scheduleData.get(i));
            	  }
              }  
             //System.out.println("Fininh Time: "+ fininhTime);
              // end doc finish time
              
              //2. check program kind
             String kind = setProgramKind(channel, program);
             
             ScheduleDetail scheduleDetail  = new ScheduleDetail(dateShow, startTime, finishTime, kind, program, detail);
             arrScheduleDetail.add(scheduleDetail);            
		}
        System.out.println(channel.getName());
		for(int j = 0;j< arrScheduleDetail.size();j++){
			ScheduleDetail sch = arrScheduleDetail.get(j);
			System.out.println("Date: "+ sch.getTimePublish().split(",")[1].trim().split(" ")[0]+ " Start: "+ sch.getStartTime()+ " " +
					" Finish: "+sch.getFinishTime()+ " Prgram: "+sch.getProgram()+"  Detail: "+sch.getDetail() +" Kind: "+sch.getKind());
		}
		
		
		scheduleChanel = new ScheduleChannel(channel, arrScheduleDetail);
		//store schedule to database
		storeScheduleOneChannel(scheduleChanel);
		//System.out.println("Finish get Schedule and Store!!");
		return scheduleChanel;
		 
	}
	
	public void storeScheduleOneChannel(ScheduleChannel scheduleChannel){
		// store schedule of one specify  Channel to database
		Channel channel = scheduleChannel.getChannel();
		ArrayList<ScheduleDetail> scheduleDetails = scheduleChannel.getScheduleDetail();
		DatabaseQueries dbq = new DatabaseQueries();
		
		for(int i=0 ;i< scheduleDetails.size();i++){
		    ScheduleDetail scheduleDetail = scheduleDetails.get(i) ;
		    /*1. insert program table */
		    //get programName
		    String category = scheduleDetail.getProgram();
		    String detail = scheduleDetail.getDetail();
		    String programName = null;
		    if(detail.indexOf("null") >-1 ){
		    	programName = category;
		    }else{
		    	programName = category + " : "+ detail;
		    }
		    //get KindID
		    String kindName = scheduleDetail.getKind();
		    int kindID = dbq.getKindID(kindName);
		    //check exist program
		    int programID = dbq.getProgramID(programName);
		    if(programID == -1){
		    	//if not exist, insert into table program
		    	dbq.insertTableProgram(programName, kindID);
		    }
		    
		    /*2. Insert schedule table*/
		    String date = scheduleDetail.getTimePublish().split(",")[1].trim().split(" ")[0].trim();
		    String dayOfWeek = scheduleDetail.getTimePublish().split(",")[0].trim();
		    String startTime = scheduleDetail.getStartTime();
		    String endTime = scheduleDetail.getFinishTime();
		    //get programID
		    programID = dbq.getProgramID(programName);			
			int channelID = dbq.getChannelID(channel.getName());
			//check exist schedule
			if(dbq.checkScheduleNotExist(channelID, programID, date)){
				dbq.insertTableSchedule(startTime, "", endTime, date, dayOfWeek, programID, channelID);
			}
			
		}
	}
    public void processGetScheduleForChannel(){
    	for(int i=0; i< arrayChanel.size();i++){
    	      getScheduleForChannel(arrayChanel.get(i));
    	}
    	System.out.println("Finish get Schedule.");
    }  
    /* end Step 3: get TV Schedule from txt data file */
     
	public static void main(String[] args){
		CrawlerVTVCab crawl = new CrawlerVTVCab();

		if(crawl.getSchedulePageInfor()){
			crawl.getChannels(); //get all channel
			crawl.storeListChannelDatabase();
			crawl.processCrawl();
			//crawl.processGetScheduleForChannel();
		}
	}
	
    /* 
     * Support function 
     * 
     * */
	public String getURL_SCHEDULE_PAGE(){
		return URL_SCHEDULE_PAGE;
	}
	public boolean checkTimeStamp(String text){
		// kiem tra xem 1 chuoi co dinh dang thoi gian ko?
		if(text.matches("(([012][0-9])||([3][01]))[/](([0][0-9])||[1][0-2])[/](19||20)[0-9][0-9]")){
			return true;
		}
		return false;
	}
	public boolean checkURLScheduleConnection(String URLSchedule){
	    	//check URL exist ??
	    	URL url;
			try {
				url = new URL(URLSchedule);
				URLConnection conn = url.openConnection();
				if(conn.getInputStream() != null){
				    return true;
				}
			} catch (Exception ex) {
				// TODO Auto-generated catch block
				//ex.printStackTrace();
			}
	       return false;
	    }

	public ArrayList<Channel> getArrayChannel(){
		return arrayChanel;
	}
	public  String nomalizeText(String text) {
        String result;
        result = text.trim();
        result = result.replaceAll("null", "");
        result = result.replaceAll("Lịch phát sóng -", "");
        result = result.replaceAll("&nbsp;", "null");
        result = result.replaceAll("&amp;", "-");
        result = result.replaceAll("  ", "");
        result = result.replaceAll("\\s+\\n", " \n");
        result = result.replaceAll("\\n+", "\n");
        result = result.trim();
        return result;
    }
    /* end Support function */
}
